package cn.ltgodm.house.config;

import cn.hutool.core.util.StrUtil;
import org.apache.hadoop.fs.FileSystem;
import org.apache.spark.sql.SparkSession;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

/**
 * @author ltgodm
 * @date 2024-06-23 13:02:54
 */
@Configuration
public class SparkConfig {
    @Value("${spark.app.name}")
    private String SPARK_APP_NAME;
    @Value("${spark.master}")
    private String SPARK_MASTER;
    @Value("${spark.local.dir}")
    private String SPARK_LOCAL_DIR;

    @Bean
    public SparkSession sparkSession() {
        SparkSession spark = SparkSession.builder()
                .appName(SPARK_APP_NAME)
                .master(SPARK_MASTER)
                .config("spark.local.dir", SPARK_LOCAL_DIR)
                .getOrCreate();
        setLogLevel(spark, "ERROR");
        return spark;
    }

    private void setLogLevel(SparkSession spark, String logLevel) {
        spark.sparkContext().setLogLevel(logLevel);
    }

}
